Introduction¶
This notebook facilitates the manual curation of sample alignment.
In [1]:
import deltascope.alignment as ut
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import h5py
import os
import re
import time
import tqdm
Setup¶
Parameters¶
In [2]:
# --------------------------------
# -------- User input ------------
# --------------------------------
param = {
'gthresh':0.5,
'scale':[1,1,1],
'microns':[0.16,0.16,0.21],
'mthresh':0.5,
'radius':20,
'comp_order':[0,2,1],
'fit_dim':['x','z'],
'deg':2,
# Don't forget to modify this with a specific sample name
'expname':'wt'
}
Directories¶
In [3]:
# --------------------------------
# -------- User input ------------
# --------------------------------
# Specify file paths to directories containing probability files
# after processing by ilastik
gfap = '.\data\ZRF1 Wild Type HZ H5 files\Prob_7-29'
at = '.\data\AT Wild Type HZ H5 files\Prob'
# Specify root directory where output will be saved
root = '.\data'
root
Out[3]:
In [4]:
# Output directory with timestamp
outname = 'Output_'+param['expname']+time.strftime("%m-%d-%H-%M",
time.localtime())
# Create output directory
outdir = os.path.join(root,outname)
os.mkdir(outdir)
Extract list of files¶
In [12]:
Dat = {}
for f in os.listdir(at):
if 'h5' in f:
num = re.findall(r'\d+',f.split('.')[0])[-1]
Dat[num] = os.path.join(at,f)
In [13]:
Dzrf = {}
for f in os.listdir(gfap):
if 'h5' in f:
num = re.findall(r'\d+',f.split('.')[0])[-1]
Dzrf[num] = os.path.join(gfap,f)
In [14]:
# Create dictionaries to contain the deltascope brain object for each sample
Dbat = {}
Dbzrf = {}
In [15]:
klist = Dzrf.keys()
klist
Out[15]:
PCA: Import raw data and perform preprocessing¶
In [16]:
%%time
for k in tqdm.tqdm(klist):
if k not in list(Dbat.keys()):
try:
Dbat[k] = ut.preprocess(Dat[k],param)
Dbzrf[k] = ut.preprocess(Dzrf[k],param,pca=Dbat[k].pcamed,
mm=Dbat[k].mm,vertex=Dbat[k].vertex)
except:
print(k,'caused an error')
else:
print(k,'already processed')
Define experiment specific functions¶
In [17]:
''' Define wrapper functions for starting and saving to minimize the number
of inputs that the user needs to type for each call of the function.'''
def start(k):
return(ut.start(k,Dbat,[Dbzrf],im=True))
def save_both(k,dfa,dfb):
ut.save_both(k,dfa,dfb,outdir,param['expname'])
'''Save model parameters for each file to a dataframe that can be
exported for later reference.'''
model = pd.DataFrame({'a':[],'b':[],'c':[]})
def save_model(k,mm,model):
row = pd.Series({'a':mm[0],'b':mm[1],'c':mm[2]},name=k)
model = model.append(row)
return(model)
'''Define a function that can both fit a model and plot it on an existing plot'''
def fit_model(axi,df,mm=None):
if mm == None:
mm = np.polyfit(df.x,df.z,2)
p = np.poly1d(mm)
xrange = np.arange(np.min(df.x),np.max(df.x))
axi.plot(xrange,p(xrange),c='m')
return(mm)
'''Take a set of points and transform to a dataframe format for ease of access.'''
def pick_pts(x1,z1,vx,vz,x2,z2):
pts = pd.DataFrame({'x':[x1,vx,x2],'z':[z1,vz,z2]})
return(pts)
Align Samples¶
In [18]:
klist
Out[18]:
In [19]:
k,df,Ldf,ax = start('101')
In [20]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [21]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [22]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [23]:
k,df,Ldf,ax = start('102')
In [24]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [25]:
df2,Ldf2,ax,p = ut.check_yz(df1,Ldf1)
In [26]:
p
Out[26]:
[a,b] $y=ax+b$
In [27]:
p
Out[27]:
In [29]:
p1 = [5/30,0]
df3,Ldf3,ax,p = ut.check_yz(df1,Ldf1,mm=p1)
In [30]:
df4,Ldf4,mm,ax = ut.ch_vertex(df3,Ldf3)
In [31]:
model = save_model(k,mm,model)
save_both(k,df4,Ldf4[0])
In [32]:
k,df,Ldf,ax = start('103')
Sample 103 appears warped in both xy and yz projections and will be discarded from processing
In [33]:
klist
Out[33]:
In [34]:
k,df,Ldf,ax = start('104')
In [ ]:
In [35]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [36]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [37]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [38]:
k,df,Ldf,ax = start('105')
In [39]:
p
Out[39]:
In [40]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [41]:
p
Out[41]:
In [42]:
p1=[.0167,-3.0]
In [43]:
p1
Out[43]:
In [44]:
df2,Ldf2,ax,p1 = ut.check_yz(df1,Ldf1)
In [45]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [46]:
k,df,Ldf,ax = start('106')
In [47]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [49]:
k,df,Ldf,ax = start('107')
save_both(k,df3,Ldf3[0])
In [50]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [51]:
k,df,Ldf,ax = start('108')
In [52]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [53]:
pts.iloc[0].x = 50
pts.iloc[1].z = 38
In [54]:
pts
Out[54]:
In [55]:
pts.iloc[0].x = -50
In [ ]:
In [56]:
ax[0,1].scatter(pts.x,pts.z,c='y')
Out[56]:
In [57]:
df2,Ldf2,ax = ut.revise_pts(df,Ldf,'z',pts=pts)
In [58]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [59]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [60]:
pts = pick_pts(-55,40,-8,-2,50,35) #these are essentially random numbers for example
In [61]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [62]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [63]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [64]:
pts = pick_pts(-53,40,-9,0,60,40)
In [65]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [66]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [67]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [68]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [69]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
sample rejected for being truncated at midline. likely improper collection that was not superficial enough to capture POC.
In [70]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [71]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [72]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [73]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [74]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [75]:
klist
Out[75]:
In [76]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [77]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
Commissure is obviously physically deformed due to pressure from coverslip and will not take parabolic curve well. skipping sample.
In [78]:
k,df,Ldf,ax = start('122')
In [79]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [80]:
k,df,Ldf,ax = start('123')
In [81]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [82]:
k,df,Ldf,ax = start('124')
In [83]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [84]:
k,df,Ldf,ax = start('125')
In [85]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [86]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [87]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [88]:
k,df,Ldf,ax = start('126')
In [89]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [90]:
p
Out[90]:
In [91]:
p1 = [-5/20, 2]
In [92]:
df1,Ldf1,ax,p1 = ut.check_yz(df,Ldf)
In [93]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf,mm=p1)
In [94]:
p1
Out[94]:
In [95]:
p2 =[-.5, 2]
In [96]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf,mm=p2)
In [97]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [98]:
k,df,Ldf,ax=start('129')
In [99]:
df,Ldf,mm,ax = ut.ch_vertex(df,Ldf)
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [100]:
k,df,Ldf,ax=start('130')
In [101]:
df,Ldf,mm,ax = ut.ch_vertex(df,Ldf)
In [102]:
model = save_model(k,mm,model)
save_both(k,df,Ldf[0])
In [103]:
k,df,Ldf,ax=start('131')
In [104]:
df,Ldf,pts,ax = ut.check_pts(df,Ldf,'z')
In [105]:
df2,Ldf2,mm,ax = ut.ch_vertex(df,Ldf)
In [106]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [107]:
k,df,Ldf,ax=start('128')
In [108]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [109]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [110]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [111]:
model
Out[111]:
In [112]:
k,df,Ldf,ax=start('127')
In [113]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [114]:
k,df,Ldf,ax=start('132')
In [115]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [116]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [117]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [118]:
k,df,Ldf,ax=start('133')
In [119]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [120]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [121]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [122]:
k,df,Ldf,ax=start('134')
In [123]:
df2,Ldf2,mm,ax = ut.ch_vertex(df,Ldf)
In [124]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [125]:
k,df,Ldf,ax=start('135')
In [126]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [127]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [128]:
df3,Ldf3,ax,p = ut.check_yz(df2,Ldf2)
In [129]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [ ]:
Wrapping up: after all samples are processed¶
Once all of the data has been processed, we want to save the model data we collected to a file for future reference.
In [ ]:
In [130]:
model.to_csv(os.path.join(outdir,'model.csv'))
Additionally, it can be helpful to export a html or pdf version of the notebook that preserves all of the plots generated during the alignment process. To do so, use the Jupyter Lab menu interface: File > Export Notebook As > HTML or PDF.
In [131]:
klist
Out[131]:
In [132]:
k,df,Ldf,ax = start('136')
In [133]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [134]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [135]:
k,df,Ldf,ax = start('137')
In [136]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [137]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [138]:
k,df,Ldf,ax = start('138')
In [139]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [140]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [141]:
k,df,Ldf,ax = start('139')
In [142]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [143]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [144]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [145]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [146]:
k,df,Ldf,ax = start('140')
sample highly disturbed in imaging structure, possibly due to tear on left of the image causing loss of data, Vertex will be highly questionable.
In [147]:
k,df,Ldf,ax = start('141')
In [148]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [149]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [150]:
k,df,Ldf,ax = start('142')
In [151]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [152]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [153]:
k,df,Ldf,ax = start('143')
In [154]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [155]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [156]:
klist
Out[156]:
In [157]:
k,df,Ldf,ax = start('105')
In [158]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [159]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [160]:
k,df,Ldf,ax = start('106')
In [161]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [162]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [163]:
k,df,Ldf,ax = start('107')
In [164]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [165]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [166]:
klist
Out[166]:
In [167]:
k,df,Ldf,ax = start('110')
In [168]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [169]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [170]:
k,df,Ldf,ax = start('115')
In [171]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [172]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [173]:
klist
Out[173]:
In [174]:
k,df,Ldf,ax = start('116')
In [175]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [176]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [177]:
k,df,Ldf,ax = start('117')
In [178]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [179]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [180]:
k,df,Ldf,ax = start('122')
In [181]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [182]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [183]:
k,df,Ldf,ax = start('123')
In [184]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [185]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [186]:
k,df,Ldf,ax = start('124')
In [187]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [188]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [189]:
k,df,Ldf,ax = start('126')
In [190]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [191]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [193]:
model.to_csv(os.path.join(outdir,'models.csv'))